package com.chance.cc.crawler.development.scripts.autohome.carSeriesVideo;

import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONObject;
import com.chance.cc.crawler.core.CrawlerEnum;
import com.chance.cc.crawler.core.CrawlerRecordContext;
import com.chance.cc.crawler.core.downloader.HttpPage;
import com.chance.cc.crawler.core.pipeline.result.CrawlerData;
import com.chance.cc.crawler.core.record.CrawlerRecord;
import com.chance.cc.crawler.core.record.CrawlerRequestRecord;
import com.chance.cc.crawler.core.script.crawler.CrawlerCommonScript;
import com.chance.cc.crawler.core.selector.Json;
import com.chance.cc.crawler.core.selector.Selectable;
import com.chance.cc.crawler.core.tags.CategoryTag;
import com.chance.cc.crawler.core.tags.KVTag;
import com.chance.cc.crawler.core.tags.LabelTag;
import com.chance.cc.crawler.core.tags.crawler.CrawlerBusinessTags;
import com.chance.cc.crawler.development.scripts.allfeild.AICCommonField;
import com.chance.cc.crawler.development.scripts.allfeild.AutoVMCommonField;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.time.DateUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.util.*;
import java.util.regex.Pattern;

import static com.chance.cc.crawler.core.CrawlerEnum.CrawlerDataType.*;
import static com.chance.cc.crawler.core.CrawlerEnum.CrawlerDataType.comment;

public class AutoHomeCarSeriesVideoCrawlerScript extends CrawlerCommonScript {
    private Logger log = LoggerFactory.getLogger(AutoHomeCarSeriesVideoCrawlerScript.class);//日志
    private static final String DOMAIN = "autohome";
    private static final String SITE = "carSeriesVideo";
    private static final String REQUEST_AGAIN_TAG = DOMAIN + "_request_again";

    //    https://car.autohome.com.cn/javascript/NewSpecCompare.js?20131010
    private static final String HomeRegulars = "https://car.autohome.com.cn/javascript/NewSpecCompare.js";//动态链接
    //    https://club.autohome.com.cn/135
    private static final String CarSeriesRegulars = "https://www.autohome.com.cn/\\d*";//车系链接
    //    https://car.autohome.com.cn/video/series-135-0-1-p1.html
    private static final String VideoRegulars = "https://car.autohome.com.cn/video/series-\\d*-0-1-p\\d*.html";//视频链接
    //    http://v.autohome.com.cn/v-2197853.html#pvareaid=2042227
    private static final String ContentRegulars = "http://v.autohome.com.cn/v-\\d*.html#pvareaid=\\d*";//视频详情1链接
    private static final String ContentsRegulars = "https://v.autohome.com.cn/v-\\d*.html#pvareaid=\\d*";//视频详情2链接
    //    http://chejiahao.autohome.com.cn/info/9234433#pvareaid=2042289
    private static final String ContentTwoRegulars = "http://chejiahao.autohome.com.cn/info/\\d*#pvareaid=\\d*";//视频详情3链接
    private static final String ContentTwosRegulars = "https://chejiahao.autohome.com.cn/info/\\d*#pvareaid=\\d*";//视频详情4链接
    //    https://reply.autohome.com.cn/api/comments/show.json?id=9057576&page=1&appid=21&count=3
    private static final String CommentRegulars = "https://reply.autohome.com.cn/api/comments/show.json\\?id=\\d*&page=\\d*&appid=\\d*&count=\\d*";//评论链接
    //https://idm.api.autohome.com.cn/api/danmu/get_config_for_pc?&biz_id=1&target_id=2196695
    private static final String DanmuRegulars = "https://idm.api.autohome.com.cn/api/danmu/get_config_for_pc\\?&biz_id=1&target_id=\\d*";//弹幕链接
    //https://reply.autohome.com.cn/api/like/getcount.json?&appid=4&liketype=1&objid=2196695
    private static final String ZanRegulars = "https://reply.autohome.com.cn/api/like/getcount.json\\?&appid=4&liketype=1&objid=\\d*";//点赞链接
    //https://reply.autohome.com.cn/api/like/listcount.json?_appid=m&appid=21&liketype=1&ids=%5B%7B%22objid%22%3A9340831%2C%22secobj%22%3A%22%22%7D%5D
    private static final String ZanTwoRegulars = "https://reply.autohome.com.cn/api/like/listcount.json\\?_appid=m&appid=21&liketype=1&ids=%5B%7B%22objid%22%3A\\d*%2C%22secobj%22%3A%22%22%7D%5D";//点赞链接2

    @Override
    public String domain() {
        return DOMAIN;
    }

    @Override
    public void initUrlRegulars() {
        addUrlRegular(HomeRegulars);
        addUrlRegular(CarSeriesRegulars);
        addUrlRegular(VideoRegulars);
        addUrlRegular(ContentRegulars);
        addUrlRegular(ContentsRegulars);
        addUrlRegular(ContentTwoRegulars);
        addUrlRegular(ContentTwosRegulars);
        addUrlRegular(CommentRegulars);

        addUrlRegular(DanmuRegulars);
        addUrlRegular(ZanRegulars);
        addUrlRegular(ZanTwoRegulars);
    }

    @Override
    public boolean crawlerCheck(CrawlerRequestRecord crawlerRecord) {
        String site = crawlerRecord.tagsCreator().bizTags().getCategoryTag().getKVTagStrVal("site");
        return SITE.equals(site);
    }

    public List<CrawlerRecord> prepareRequest(CrawlerRequestRecord crawlerRequestRecord, List<CrawlerRequestRecord> supportSourceRecords) {
        List<CrawlerRecord> prepareLinks = new ArrayList<>();
        for (CrawlerRequestRecord supportSourceRecord : supportSourceRecords) {
            String url = supportSourceRecord.getHttpRequest().getUrl();
            if (url.contains("keys")) {
                this.initKeyWord(crawlerRequestRecord, supportSourceRecord);
            }
        }
        return prepareLinks;
    }

    //取出代表不同的车系key
    private void initKeyWord(CrawlerRequestRecord crawlerRequestRecord, CrawlerRequestRecord supportSourceRecord) {
        HttpPage internalDownloadPage = supportSourceRecord.getInternalDownloadPage();
        Json json = internalDownloadPage.getJson();
        String msg = json.jsonPath($_type + ".msg").get();
        if (!"success".equals(msg)) {
            log.error("keyword page [{}] download error!", internalDownloadPage.getRequest().getUrl());
            return;
        }
        List<String> keys = new ArrayList<>();
        List<String> all = json.jsonPath($_type + ".content").all();
        for (String data : all) {
            JSONObject jsonObject = JSONObject.parseObject(data);
            String keyword = jsonObject.getString("keyword");
            keys.add(keyword);
        }
        crawlerRequestRecord.tagsCreator().bizTags().addCustomKV("keys", keys);
    }

    @Override
    public List<CrawlerRequestRecord> parseLinks(CrawlerRequestRecord crawlerRecord, HttpPage page) {
        List<CrawlerRequestRecord> parsedLinks = new ArrayList();
        //        判断页面是否加载成功
        String requestUrl = crawlerRecord.getHttpRequest().getUrl();
        if (page.getStatusCode() != 200 || (!page.isDownloadSuccess())) {
            log.error("download page url == {null} error status is {}", requestUrl, page.getStatusCode());
            if (page.getStatusCode() != 404) {//如果没有成功的下载  进行重新下载
                this.requestAgainCrawlerRecord(parsedLinks, crawlerRecord);
                crawlerRecord.setNeedWashPage(false);
                return parsedLinks;
            }
        }
        if (crawlerRecord.getHttpRequest().getUrl().matches(HomeRegulars)) {
            homeRequest(crawlerRecord, page, parsedLinks);//解析动态车系
        } else if (crawlerRecord.getHttpRequest().getUrl().matches(CarSeriesRegulars)) {
            carSeriesRequest(crawlerRecord, page, parsedLinks);//解析车系
        } else if (crawlerRecord.getHttpRequest().getUrl().matches(VideoRegulars)) {
            videoRequest(crawlerRecord, page, parsedLinks);//解析视频列表和评论
        } else if (crawlerRecord.getHttpRequest().getUrl().matches(DanmuRegulars)) {
            danmuRequest(crawlerRecord, page, parsedLinks);//解析视频列表和评论
        } else if (crawlerRecord.getHttpRequest().getUrl().matches(ZanRegulars) || crawlerRecord.getHttpRequest().getUrl().matches(ZanTwoRegulars)) {
            zanRequest(crawlerRecord, page, parsedLinks);//解析视频列表和评论
        }
        return parsedLinks;
    }

    //解析动态车系
//    private void homeRequest(CrawlerRequestRecord crawlerRecord, HttpPage page, List<CrawlerRequestRecord> parsedLinks) {
//        Map<String, String> carMap = new HashMap<>();
//        String rawText = page.getRawText();//获取解析url后的字符串
//        String json = rawText.substring(21, rawText.length() - 3);//获取json
//        List<String> number = new Json(json).jsonPath($_type + "..I").all();
//        List<String> carName = new Json(json).jsonPath($_type + "..N").all();
//        for (int i = 0; i < number.size(); i++) {
//            carMap.put(carName.get(i), number.get(i));
//        }
//        KVTag keys = crawlerRecord.tagsCreator().bizTags().getCategoryTag().getKVTag("keys");
//        List<String> name = (List<String>) keys.getVal();
//        ArrayList<String> list = new ArrayList<>();
//        for (int i = 0; i < name.size(); i++) {//
//            boolean discontinued = name.get(i).contains("停售");//直接进入停售页面
//            String nameNumber = carMap.get(name.get(i));//品牌转换为id数字
//            if (nameNumber == null) {
//                list.add(name.get(i));
//                log.error("没有这个车系（进口，停售等加上字段 例：开瑞K50EV(停售)）" + name.get(i));
//            } else {
//                if (discontinued) {
//                    String discontinuedUrl = "https://www.autohome.com.cn/" + nameNumber + "/sale.html";//得到车系页面
//                    CrawlerRequestRecord discontinuedRecord = CrawlerRequestRecord.builder()//解析停售
//                            .itemPageRequest(crawlerRecord)
//                            .recordKey(discontinuedUrl)
//                            .httpUrl(discontinuedUrl)
//                            .releaseTime(System.currentTimeMillis())
//                            .resultLabelTag(article)
//                            .copyBizTags()
//                            .build();
//                    parsedLinks.add(discontinuedRecord);
//                } else {
//                    String url = "https://www.autohome.com.cn/" + nameNumber;//得到车系页面
//                    CrawlerRequestRecord record = CrawlerRequestRecord.builder()
//                            .itemPageRequest(crawlerRecord)
//                            .httpUrl(url)
//                            .releaseTime(Long.valueOf(System.currentTimeMillis()))
//                            .copyBizTags()
//                            .build();
//                    parsedLinks.add(record);
//                }
//            }
//        }
//    }

    //    解析动态车系
    private void homeRequest(CrawlerRequestRecord crawlerRecord, HttpPage page, List<CrawlerRequestRecord> parsedLinks) {
        if (crawlerRecord.tagsCreator().bizTags().getCategoryTag().getKVTagListVal("keys") != null) {
            KVTag keys = crawlerRecord.tagsCreator().bizTags().getCategoryTag().getKVTag("keys");
            List<String> name = (List<String>) keys.getVal();
            if (name.size() > 0) {
                for (int i = 0; i < name.size(); i++) {//name.size()
                    if (name.get(i) != null && name.get(i) != "") {
                        String url = "https://www.autohome.com.cn/" + name.get(i);//得到车系页面
                        CrawlerRequestRecord record = CrawlerRequestRecord.builder()
                                .itemPageRequest(crawlerRecord)
                                .httpUrl(url)
                                .releaseTime(Long.valueOf(System.currentTimeMillis()))
                                .copyBizTags()
                                .copyResultTags()
                                .build();
                        parsedLinks.add(record);
                    }
                }
            }
        }
    }

    //解析车系
    private void carSeriesRequest(CrawlerRequestRecord crawlerRecord, HttpPage page, List<CrawlerRequestRecord> parsedLinks) {
        String brandName = page.getHtml().xpath("//div[@class=\"container athm-crumb\"]/a[2]/text()").get();//品牌名
        String brandUrl = "https:" + page.getHtml().xpath("//div[@class=\"container athm-crumb\"]/a[2]/@href").get();//品牌url
        String brandId = Pattern.compile("[^0-9]").matcher(brandUrl).replaceAll("").trim();//品牌id
        String carSeriesUrl = page.getRequest().getUrl();//车系url
        String carSeriesName = page.getHtml().xpath("//h1/a").get();//车系名
        String carSeriesId = Pattern.compile("[^0-9]").matcher(carSeriesUrl).replaceAll("").trim();//车系id
        HashMap<String, String> seriesMap = new HashMap<>();
        seriesMap.put(AutoVMCommonField.Field_Series_name, carSeriesName);//车系名
        seriesMap.put(AutoVMCommonField.Field_Series_url, carSeriesUrl);//车系url
        seriesMap.put(AutoVMCommonField.Field_Series_id, carSeriesId);//车系url
        crawlerRecord.tagsCreator().bizTags().addCustomKV(AutoVMCommonField.Tag_Field_Series, seriesMap);//车系
        HashMap<String, String> brandMap = new HashMap<>();
        brandMap.put(AutoVMCommonField.Field_Brand_name, brandName);//品牌名
        brandMap.put(AutoVMCommonField.Field_Brand_url, brandUrl);//品牌url
        brandMap.put(AutoVMCommonField.Field_Brand_id, brandId);//品牌url
        crawlerRecord.tagsCreator().bizTags().addCustomKV(AutoVMCommonField.Tag_Field_Brand, brandMap);//品牌
        //https://car.autohome.com.cn/video/series-135-0-1-p1.html
        String articleId = Pattern.compile("[^0-9]").matcher(carSeriesUrl).replaceAll(" ").replaceAll("\\s+", " ").trim().split(" ")[0];
        String videoUrl = "https://car.autohome.com.cn/video/series-" + articleId + "-0-1-p1.html";
        CrawlerRequestRecord videoRecord = CrawlerRequestRecord.builder()//解析视频页
                .itemPageRequest(crawlerRecord)
                .recordKey(videoUrl)
                .httpUrl(videoUrl)
                .releaseTime(System.currentTimeMillis())
                .copyBizTags()
                .copyResultTags()
                .build();
        videoRecord.tagsCreator().bizTags().addCustomKV(AICCommonField.Tag_Site_Info, videoUrl);
        parsedLinks.add(videoRecord);
    }

    //解析视频列表
    private void videoRequest(CrawlerRequestRecord crawlerRecord, HttpPage page, List<CrawlerRequestRecord> parsedLinks) {
//        下一页
        String nextStr = page.getHtml().xpath("//a[./text()=\"下一页\"]/@href").get();
        if (nextStr != null || nextStr != "") {
            String nextPageUrl = "https://car.autohome.com.cn" + nextStr;//翻页
            CrawlerRequestRecord turnPageRequest = CrawlerRequestRecord.builder()
                    .turnPageRequest(crawlerRecord)
                    .recordKey(nextPageUrl)
                    .httpUrl(nextPageUrl)
                    .releaseTime(System.currentTimeMillis())
                    .copyBizTags()
                    .copyResultTags()
                    .build();
            parsedLinks.add(turnPageRequest);
        }
//        明细页解析
        List<Selectable> nodes = page.getHtml().xpath("//ul[@class=\"videocont\"]/li").nodes();//所有视频
        for (Selectable node : nodes) {
            String comments = "0";
            String commentStr = node.xpath(".//div[@class=\"videocont-read\"]/span[2]/text()").get();
            if (commentStr != null && commentStr != "") {
                comments = node.xpath(".//div[@class=\"videocont-read\"]/span[2]/text()").get();//评论数
            }
            String views = node.xpath(".//div[@class=\"videocont-read\"]/span[1]/text()").get();//浏览量
            String videoUrl = node.xpath(".//a/@href").get();//视频url
            if (videoUrl != null && videoUrl != "") {
                String videoId = Pattern.compile("[^0-9]").matcher(videoUrl).replaceAll(" ").replaceAll("\\s+", " ").trim().split(" ")[0];//url的关键字
                if (videoId != null) {
                    if (videoUrl.matches(ContentRegulars) || videoUrl.matches(ContentsRegulars)) {
                        //https://idm.api.autohome.com.cn/api/danmu/get_config_for_pc?&biz_id=1&target_id=2196695
                        String danmuUrl = "https://idm.api.autohome.com.cn/api/danmu/get_config_for_pc?&biz_id=1&target_id=" + videoId;//弹幕url
                        //https://reply.autohome.com.cn/api/like/getcount.json?&appid=4&liketype=1&objid=2196695
                        String zanUrl = "https://reply.autohome.com.cn/api/like/getcount.json?&appid=4&liketype=1&objid=" + videoId;//赞url

                        CrawlerRequestRecord danmuRecord = CrawlerRequestRecord.builder()//解析赞
                                .itemPageRequest(crawlerRecord)
                                .recordKey(danmuUrl)
                                .httpUrl(danmuUrl)
                                .releaseTime(System.currentTimeMillis())
                                .copyBizTags()
                                .copyResultTags()
                                .build();
                        danmuRecord.tagsCreator().bizTags().addCustomKV(AICCommonField.Field_I_Comments, comments);
                        danmuRecord.tagsCreator().bizTags().addCustomKV(AICCommonField.Field_I_Views, views);
                        danmuRecord.tagsCreator().bizTags().addCustomKV("zanUrl", zanUrl);
                        danmuRecord.tagsCreator().bizTags().addCustomKV(AICCommonField.Field_Urls, videoUrl);
                        parsedLinks.add(danmuRecord);
                    }
                    if (videoUrl.matches(ContentTwoRegulars) || videoUrl.matches(ContentTwosRegulars)) {
                        //https://reply.autohome.com.cn/api/like/listcount.json?_appid=m&appid=21&liketype=1&ids=%5B%7B%22objid%22%3A9340831%2C%22secobj%22%3A%22%22%7D%5D
                        String zanUrl = "https://reply.autohome.com.cn/api/like/listcount.json?_appid=m&appid=21&liketype=1&ids=%5B%7B%22objid%22%3A" + videoId + "%2C%22secobj%22%3A%22%22%7D%5D";//赞url

                        CrawlerRequestRecord zanRecord = CrawlerRequestRecord.builder()//解析赞
                                .itemPageRequest(crawlerRecord)
                                .recordKey(zanUrl)
                                .httpUrl(zanUrl)
                                .releaseTime(System.currentTimeMillis())
                                .copyBizTags()
                                .copyResultTags()
                                .build();
                        zanRecord.tagsCreator().bizTags().addCustomKV(AICCommonField.Field_I_Comments, comments);
                        zanRecord.tagsCreator().bizTags().addCustomKV(AICCommonField.Field_I_Views, views);
                        zanRecord.tagsCreator().bizTags().addCustomKV(AICCommonField.Field_Urls, videoUrl);
                        parsedLinks.add(zanRecord);
                    }
                }
            }
        }
    }

    //解析弹幕
    private void danmuRequest(CrawlerRequestRecord crawlerRecord, HttpPage page, List<CrawlerRequestRecord> parsedLinks) {
        String danmuVal = "0";
        Selectable selectable = page.getJson().jsonPath($_type + "..msg_count");
        if (selectable != null || "".equals(selectable)) {
            danmuVal = page.getJson().jsonPath($_type + "..msg_count").get();//弹幕
        }
        String zanUrl = crawlerRecord.tagsCreator().bizTags().getCategoryTag().getKVTagStrVal("zanUrl");
        if (zanUrl != null && zanUrl != "") {
            try {
                CrawlerRequestRecord danmuRecord = CrawlerRequestRecord.builder()//解析弹幕
                        .itemPageRequest(crawlerRecord)
                        .recordKey(zanUrl)
                        .httpUrl(zanUrl)
                        .releaseTime(System.currentTimeMillis())
                        .copyBizTags()
                        .copyResultTags()
                        .build();
                danmuRecord.tagsCreator().bizTags().addCustomKV(AICCommonField.Field_I_Bullet_Chat, danmuVal);
                parsedLinks.add(danmuRecord);
            } catch (Exception e) {
                log.error(e.getMessage(), e);
            }
        }
    }

    //解析赞
    private void zanRequest(CrawlerRequestRecord crawlerRecord, HttpPage page, List<CrawlerRequestRecord> parsedLinks) {
        try {
            String videoUrl = crawlerRecord.tagsCreator().bizTags().getCategoryTag().getKVTagStrVal(AICCommonField.Field_Urls);
            String comments = crawlerRecord.tagsCreator().bizTags().getCategoryTag().getKVTagStrVal(AICCommonField.Field_I_Comments);
            String zanVal = "0";
            Selectable selectable = page.getJson().jsonPath($_type + "..count");
            if (selectable != null || "".equals(selectable)) {
                zanVal = page.getJson().jsonPath($_type + "..count").get();//点赞
            }
            if (videoUrl != null && videoUrl != "") {
                String videoId = Pattern.compile("[^0-9]").matcher(videoUrl).replaceAll(" ").replaceAll("\\s+", " ").trim().split(" ")[0];//url的关键字
                CrawlerRequestRecord videoRecord = CrawlerRequestRecord.builder()//解析视频
                        .itemPageRequest(crawlerRecord)
                        .recordKey(videoUrl)
                        .httpUrl(videoUrl)
                        .releaseTime(System.currentTimeMillis())
                        .copyBizTags()
                        .copyResultTags()
                        .needWashed(true)
                        .build();
                videoRecord.tagsCreator().bizTags().addCustomKV(AICCommonField.Field_I_Likes, zanVal);
                parsedLinks.add(videoRecord);

                if (Integer.parseInt(comments) > 0) {
                    //      https://reply.autohome.com.cn/api/comments/show.json?id=9057576&page=1&appid=21&count=3 评论url
                    String commentUrl = "https://reply.autohome.com.cn/api/comments/show.json?id=" + videoId + "&page=1&appid=21&count=" + comments;
                    String commentTags = crawlerRecord.tagsCreator().bizTags().getCategoryTag().getKVTagStrVal("commentTags");
                    if ("comment_Tags".equals(commentTags)) {
                        KVTag filterInfoTag = crawlerRecord.tagsCreator().bizTags().getCategoryTag().getKVTag("comment_record_filter_info");
                        if (filterInfoTag != null) {
                            CrawlerRecord filterInfoRecord = JSON.parseObject((String) filterInfoTag.getVal(), CrawlerRecord.class);
                            CrawlerRequestRecord commentRecord = CrawlerRequestRecord.builder()//解析视频评论
                                    .itemPageRequest(crawlerRecord)
                                    .recordKey(commentUrl)
                                    .httpUrl(commentUrl)
                                    .releaseTime(System.currentTimeMillis())
                                    .copyBizTags()
                                    .copyResultTags()
                                    .notFilterRecord()
                                    .needWashed(true)
                                    .build();
                            commentRecord.tagsCreator().bizTags().addCustomKV(AICCommonField.Field_Urls, videoUrl);
                            commentRecord.setFilter(filterInfoRecord.getFilter());
                            commentRecord.setFilterInfos(filterInfoRecord.getFilterInfos());
                            parsedLinks.add(commentRecord);
                        }
                    }
                }
            }
        } catch (Exception e) {
            log.error(e.getMessage(), e);
        }
    }

    @Override
    public List<CrawlerData> washPage(CrawlerRequestRecord crawlerRecord, HttpPage page) {
        crawlerRecord.tagsCreator().bizTags().addCustomKV("keys","");
        List<CrawlerData> crawlerDataList = new ArrayList<>();
        if (crawlerRecord.getHttpRequest().getUrl().matches(ContentRegulars) || crawlerRecord.getHttpRequest().getUrl().matches(ContentsRegulars) || crawlerRecord.getHttpRequest().getUrl().matches(ContentTwosRegulars) || crawlerRecord.getHttpRequest().getUrl().matches(ContentTwoRegulars)) {//视频互动量
            if (crawlerRecord.tagsCreator().resultTags().hasDataType(interaction)) {
                crawlerDataList.addAll(videoInteractionWash(crawlerRecord, page));
            }
        }
        if (crawlerRecord.getHttpRequest().getUrl().matches(ContentRegulars) || crawlerRecord.getHttpRequest().getUrl().matches(ContentsRegulars)) {//文章1
            if (crawlerRecord.tagsCreator().resultTags().hasDataType(article)) {
                crawlerDataList.addAll(videoWash(crawlerRecord, page));
            }
            if (crawlerRecord.tagsCreator().resultTags().hasDataType(article)) {//视频链接
                crawlerDataList.addAll(articleListWash(crawlerRecord, page));
            }
        }
        if (crawlerRecord.getHttpRequest().getUrl().matches(ContentTwoRegulars) || crawlerRecord.getHttpRequest().getUrl().matches(ContentTwosRegulars)) {//文章2
            if (crawlerRecord.tagsCreator().resultTags().hasDataType(article)) {
                crawlerDataList.addAll(videoTwoWash(crawlerRecord, page));
            }
            if (crawlerRecord.tagsCreator().resultTags().hasDataType(article)) {//视频链接
                crawlerDataList.addAll(articleListWash(crawlerRecord, page));
            }
        }
        if (crawlerRecord.getHttpRequest().getUrl().matches(CommentRegulars)) {//评论
            if (crawlerRecord.tagsCreator().resultTags().hasDataType(comment)) {//评论
                crawlerDataList.addAll(commentWash(crawlerRecord, page));
            }
            if (crawlerRecord.tagsCreator().resultTags().hasDataType(comment)) {//评论的回复
                crawlerDataList.addAll(replyWash(crawlerRecord, page));
            }
            if (crawlerRecord.tagsCreator().resultTags().hasDataType(interaction)) {//评论互动量
                crawlerDataList.addAll(commentInteractionWash(crawlerRecord, page));
            }
        }
        return crawlerDataList;
    }

    //清洗文章链接
    public List<CrawlerData> articleListWash(CrawlerRequestRecord crawlerRecord, HttpPage page) {
        log.info("清洗链接");

        List<CrawlerData> crawlerArticleDataList = new ArrayList<>();
        try {
            crawlerRecord.getTags().clear();//去掉之前的标签
            String videoUrl = page.getRequest().getUrl();//视频url

            long time = 0;
            if (crawlerRecord.getHttpRequest().getUrl().matches(ContentRegulars) || crawlerRecord.getHttpRequest().getUrl().matches(ContentsRegulars)) {
                String releaseTime = page.getHtml().xpath("//span[@class=\"date\"]").get();//时间
                if (releaseTime == null) {
                    log.error("页面超时，时间为空{}", videoUrl);
                }
                time = DateUtils.parseDate(releaseTime, "yyyy年MM月dd日").getTime();
            }
            if (crawlerRecord.getHttpRequest().getUrl().matches(ContentTwoRegulars) || crawlerRecord.getHttpRequest().getUrl().matches(ContentTwosRegulars)) {//文章2
                String releaseTime = page.getHtml().xpath("//div[@class=\"articleTag\"]/span[3]").get();//时间
                if (releaseTime == null) {
                    log.error("页面超时，时间为空{}", videoUrl);
                }
                time = DateUtils.parseDate(releaseTime, "yyyy-MM-dd").getTime();
            }
            String[] videoId = Pattern.compile("[^0-9]").matcher(videoUrl).replaceAll(" ").replaceAll("\\s+", " ").trim().split(" ");//url的关键字
            CrawlerData crawlerData = CrawlerData.builder()
                    .data(crawlerRecord, page)
                    .dataId(StringUtils.joinWith("-", crawlerRecord.getDomain(), article.enumVal(), videoId[0]))
                    .resultLabelTag(CrawlerEnum.CrawlerDataType.valueOf("article"))
                    .url(videoUrl)
                    .releaseTime(time)
                    .addContentKV("releaseTimeToLong", String.valueOf(time))
                    .addContentKV("articleUrl", videoUrl)
                    .addContentKV(AICCommonField.Field_Author, "视频")
                    .flowInPipelineTag("redis")
                    .build();
            crawlerData.setFilterPipelineResult(true);
            crawlerArticleDataList.add(crawlerData);
        } catch (Exception e) {
            log.error(e.getMessage(), e);
        }
        return crawlerArticleDataList;
    }


    //  视频互动量清洗
    private List<CrawlerData> videoInteractionWash(CrawlerRequestRecord crawlerRecord, HttpPage page) {
        log.info("视频互动量清洗");

        List<CrawlerData> crawlerArticleDataList = new ArrayList<>();
        if (page.getRequest() != null) {
            String comments = crawlerRecord.tagsCreator().bizTags().getCategoryTag().getKVTagStrVal(AICCommonField.Field_I_Comments);
            String views = crawlerRecord.tagsCreator().bizTags().getCategoryTag().getKVTagStrVal(AICCommonField.Field_I_Views);
            String like = crawlerRecord.tagsCreator().bizTags().getCategoryTag().getKVTagStrVal(AICCommonField.Field_I_Likes);
            String danmu = crawlerRecord.tagsCreator().bizTags().getCategoryTag().getKVTagStrVal(AICCommonField.Field_I_Bullet_Chat);
            if (danmu == null) {
                danmu = "无";
            }
            String videoUrl = crawlerRecord.tagsCreator().bizTags().getCategoryTag().getKVTagStrVal(AICCommonField.Field_Urls);//文章url

            try {
                long time = 0;
                if (crawlerRecord.getHttpRequest().getUrl().matches(ContentRegulars) || crawlerRecord.getHttpRequest().getUrl().matches(ContentsRegulars)) {
                    String releaseTime = page.getHtml().xpath("//span[@class=\"date\"]").get();//时间
                    if (releaseTime == null) {
                        log.error("页面超时，时间为空{}", videoUrl);
                    }
                    time = DateUtils.parseDate(releaseTime, "yyyy年MM月dd日").getTime();
                }
                if (crawlerRecord.getHttpRequest().getUrl().matches(ContentTwoRegulars) || crawlerRecord.getHttpRequest().getUrl().matches(ContentTwosRegulars)) {//文章2
                    String releaseTime = page.getHtml().xpath("//div[@class=\"articleTag\"]/span[3]").get();//时间
                    if (releaseTime == null) {
                        log.error("页面超时，时间为空{}", videoUrl);
                    }
                    time = DateUtils.parseDate(releaseTime, "yyyy-MM-dd").getTime();
                }

                String videoId = Pattern.compile("[^0-9]").matcher(videoUrl).replaceAll(" ").replaceAll("\\s+", " ").trim().split(" ")[0];//url的关键字

                CrawlerData crawlerData = CrawlerData.builder()
                        .data(crawlerRecord, page)
                        .dataId(StringUtils.joinWith("-", crawlerRecord.getDomain(), interaction.enumVal(), videoId))
                        .parentId(StringUtils.joinWith("-", crawlerRecord.getDomain(), article.enumVal(), videoId))
                        .resultLabelTag(CrawlerEnum.CrawlerDataType.valueOf("interaction"))
                        .url(videoUrl)
                        .releaseTime(time)
                        .addContentKV(AICCommonField.Field_I_Views, views)
                        .addContentKV(AICCommonField.Field_I_Comments, comments)
                        .addContentKV(AICCommonField.Field_I_Likes, like)
                        .addContentKV(AICCommonField.Field_I_Bullet_Chat, danmu)
                        .requestLabelTag(CrawlerEnum.CrawlerRequestType.result)
                        .requestLabelTag(CrawlerEnum.CrawlerRequestType.filter)
                        .flowInPipelineTag("kafka")
                        .build();
                crawlerData.setFilterPipelineResult(true);
                crawlerArticleDataList.add(crawlerData);
            } catch (Exception e) {
                log.error(e.getMessage(), e);
            }
        }

        return crawlerArticleDataList;
    }

    //解析视频1
    public List<CrawlerData> videoWash(CrawlerRequestRecord crawlerRecord, HttpPage page) {
        log.info("清洗视频1");

        List<CrawlerData> crawlerArticleDataList = new ArrayList<>();
        if (page.getRequest() != null) {
            String videoUrl = page.getRequest().getUrl();//视频url
            String releaseTime = page.getHtml().xpath("//span[@class=\"date\"]").get();//时间
            String title = page.getHtml().xpath("//h1").get();//标题
            String content = page.getHtml().xpath("//div[@class=\"video-msg__text\"]/p/span").get();//简介
            String followStr = page.getHtml().xpath("//span[@id=\"fans\"]").get();//粉丝数
            String follow = numberBefore(followStr);
            String authorName = page.getHtml().xpath("//div[@class=\"video-channel\"]/a[2]").get().trim();//作者名
            String adminUrl = page.getHtml().xpath("//div[@class=\"video-channel\"]/a[1]/@href").get();//作者url
            String authorId = Pattern.compile("[^0-9]").matcher(adminUrl).replaceAll("").trim();//作者id
            List<String> tag = page.getHtml().xpath("//div[@class=\"video-msg__text\"]/p/a/text()").all();//标签
            StringBuffer tags = new StringBuffer();//拼接标签
            for (String s : tag) {
                tags.append(s).append(" ");
            }
            String videoId = Pattern.compile("[^0-9]").matcher(videoUrl).replaceAll(" ").replaceAll("\\s+", " ").trim().split(" ")[0];//url的关键字
            try {
                CrawlerData crawlerData = CrawlerData.builder()
                        .data(crawlerRecord, page)
                        .dataId(StringUtils.joinWith("-", crawlerRecord.getDomain(), article.enumVal(), videoId))
                        .resultLabelTag(CrawlerEnum.CrawlerDataType.valueOf("article"))
                        .url(videoUrl)
                        .releaseTime(DateUtils.parseDate(releaseTime, "yyyy年MM月dd日").getTime())
                        .addContentKV(AICCommonField.Field_Content, content.trim())
                        .addContentKV(AICCommonField.Field_Title, title.trim())
                        .addContentKV(AICCommonField.Field_Author, authorName)
                        .addContentKV(AICCommonField.Field_Author_Id, authorId)
                        .addContentKV(AutoVMCommonField.Tag_Field_Topic_Type, tags.toString().trim())
                        .addContentKV(AutoVMCommonField.Field_Author_Follows, follow)
                        .flowInPipelineTag("kafka")
                        .build();
                crawlerData.setFilterPipelineResult(true);
                crawlerArticleDataList.add(crawlerData);
            } catch (Exception e) {
                log.error(e.getMessage(), e);
            }
        }
        return crawlerArticleDataList;
    }

    //解析视频2
    public List<CrawlerData> videoTwoWash(CrawlerRequestRecord crawlerRecord, HttpPage page) {
        log.info("清洗视频2");

        List<CrawlerData> crawlerArticleDataList = new ArrayList<>();
        if (page.getRequest() != null) {
            String videoUrl = page.getRequest().getUrl();//视频url
            String releaseTime = page.getHtml().xpath("//div[@class=\"articleTag\"]/span[3]").get();//时间
            String title = page.getHtml().xpath("//div[@class=\"title\"]").get();//标题
            String content = page.getHtml().xpath("//p[@class=\"text\"]").get();//简介
            String followStr = page.getHtml().xpath("//div[@class=\"num\"]/span[1]").get();//粉丝数
            String follow = numberBefore(followStr);
            String authorName = page.getHtml().xpath("//div[@class=\"articleTag\"]/span[1]").get();//作者名
            String adminUrl = page.getHtml().xpath("//div[@class=\"cover\"]/a/@href").get();//作者url
            String authorId = Pattern.compile("[^0-9]").matcher(adminUrl).replaceAll("");//作者id

            List<String> tag = page.getHtml().xpath("//div[@class=\"tagBotV\"]//a/span").all();//标签
            StringBuffer tags = new StringBuffer();//拼接标签
            for (String s : tag) {
                tags.append(s).append(" ");
            }
            String videoId = Pattern.compile("[^0-9]").matcher(videoUrl).replaceAll(" ").replaceAll("\\s+", " ").trim().split(" ")[0];//url的关键字
            try {
                CrawlerData crawlerData = CrawlerData.builder()
                        .data(crawlerRecord, page)
                        .dataId(StringUtils.joinWith("-", crawlerRecord.getDomain(), article.enumVal(), videoId))
                        .resultLabelTag(CrawlerEnum.CrawlerDataType.valueOf("article"))
                        .url(videoUrl)
                        .releaseTime(DateUtils.parseDate(releaseTime, "yyyy-MM-dd").getTime())
                        .addContentKV(AICCommonField.Field_Content, content.trim())
                        .addContentKV(AICCommonField.Field_Title, title.trim())
                        .addContentKV(AICCommonField.Field_Author, authorName)
                        .addContentKV(AICCommonField.Field_Author_Id, authorId)
                        .addContentKV(AutoVMCommonField.Tag_Field_Topic_Type, tags.toString())
                        .addContentKV(AutoVMCommonField.Field_Author_Follows, follow)
                        .flowInPipelineTag("kafka")
                        .build();
                crawlerData.setFilterPipelineResult(true);
                crawlerArticleDataList.add(crawlerData);
            } catch (Exception e) {
                log.error(e.getMessage(), e);
            }
        }
        return crawlerArticleDataList;
    }

    //  评论清洗
    private Collection<? extends CrawlerData> commentWash(CrawlerRequestRecord crawlerRecord, HttpPage page) {
        log.info("清洗评论");

        List<CrawlerData> crawlerArticleDataList = new ArrayList<>();
        if (page.getJson() != null || "".equals(page.getJson())) {
            List<Selectable> nodes = page.getJson().jsonPath($_type + ".commentlist[*]").nodes();
            for (Selectable node : nodes) {
                try {
                    String content = new Json(node.get()).jsonPath(".RContent").get();//评论内容
                    String timeStr = new Json(node.get()).jsonPath(".RReplyDate").get();//回复时间
                    Long time = Long.valueOf(Pattern.compile("[^0-9]").matcher(timeStr).replaceAll(" ").replaceAll("\\s+", " ").trim().split(" ")[0]);
                    String user = new Json(node.get()).jsonPath(".RMemberName").get();//评论用户
                    String userId = new Json(node.get()).jsonPath(".RMemberId").get();//评论用户id
                    String commentId = new Json(node.get()).jsonPath(".ReplyId").get();//评论id

                    String videoUrl = crawlerRecord.tagsCreator().bizTags().getCategoryTag().getKVTagStrVal(AICCommonField.Field_Urls);//文章url
                    String videoId = Pattern.compile("[^0-9]").matcher(videoUrl).replaceAll(" ").replaceAll("\\s+", " ").trim().split(" ")[0];//url的关键字

                    CrawlerData crawlerData = CrawlerData.builder()
                            .data(crawlerRecord, page)
                            .dataId(StringUtils.joinWith("-", crawlerRecord.getDomain(), comment.enumVal(), commentId))
                            .parentId(StringUtils.joinWith("-", crawlerRecord.getDomain(), article.enumVal(), videoId))
                            .resultLabelTag(CrawlerEnum.CrawlerDataType.valueOf("comment"))
                            .url(videoUrl)
                            .releaseTime(time)
                            .addContentKV(AICCommonField.Field_Content, content)
                            .addContentKV(AICCommonField.Field_Author, user)
                            .addContentKV(AICCommonField.Field_Author_Id, userId)
                            .requestLabelTag(CrawlerEnum.CrawlerRequestType.result)
                            .requestLabelTag(CrawlerEnum.CrawlerRequestType.filter)
                            .flowInPipelineTag("kafka")
                            .build();
                    crawlerData.setFilterPipelineResult(true);
                    crawlerArticleDataList.add(crawlerData);
                } catch (Exception e) {
                    log.error(e.getMessage(), e);
                }
            }
        }
        return crawlerArticleDataList;
    }

    //  评论互动量
    private Collection<? extends CrawlerData> commentInteractionWash(CrawlerRequestRecord crawlerRecord, HttpPage page) {
        log.info("清洗评论互动量");

        List<CrawlerData> crawlerArticleDataList = new ArrayList<>();
        List<Selectable> nodes = page.getJson().jsonPath($_type + ".commentlist[*]").nodes();//所有评论
        for (Selectable node : nodes) {
            try {
                String like = new Json(node.get()).jsonPath(".RUp").get();//顶
                String timeStr = new Json(node.get()).jsonPath(".RReplyDate").get();//回复时间
                Long time = Long.valueOf(Pattern.compile("[^0-9]").matcher(timeStr).replaceAll(" ").replaceAll("\\s+", " ").trim().split(" ")[0]);
                String userId = new Json(node.get()).jsonPath(".RMemberId").get();//用户id
                String commentId = new Json(node.get()).jsonPath(".ReplyId").get();//评论id
                String videoUrl = crawlerRecord.tagsCreator().bizTags().getCategoryTag().getKVTagStrVal(AICCommonField.Field_Urls);//文章url

                CrawlerData crawlerData = CrawlerData.builder()
                        .data(crawlerRecord, page)
                        .dataId(StringUtils.joinWith("-", crawlerRecord.getDomain(), interaction.enumVal(), commentId))
                        .parentId(StringUtils.joinWith("-", crawlerRecord.getDomain(), comment.enumVal(), userId))
                        .resultLabelTag(CrawlerEnum.CrawlerDataType.valueOf("comment"))
                        .url(videoUrl)
                        .releaseTime(time)
                        .addContentKV(AICCommonField.Field_I_Likes, like)
                        .requestLabelTag(CrawlerEnum.CrawlerRequestType.result)
                        .requestLabelTag(CrawlerEnum.CrawlerRequestType.filter)
                        .flowInPipelineTag("kafka")
                        .build();
                crawlerData.setFilterPipelineResult(true);
                crawlerArticleDataList.add(crawlerData);
            } catch (Exception e) {
                log.error(e.getMessage(), e);
            }
        }
        return crawlerArticleDataList;
    }

    //  评论回复
    private Collection<? extends CrawlerData> replyWash(CrawlerRequestRecord crawlerRecord, HttpPage page) {
        log.info("评论回复");

        List<CrawlerData> crawlerArticleDataList = new ArrayList<>();
        List<Selectable> nodes = page.getJson().jsonPath($_type + ".commentlist[*]").nodes();
        for (Selectable node : nodes) {
            String userId2 = new Json(node.get()).jsonPath(".Quote.RMemberId").get();//原评论用户id
            if (userId2 != null) {
                try {
                    String content1 = new Json(node.get()).jsonPath(".RContent").get();//回复内容
                    String timeStr = new Json(node.get()).jsonPath(".RReplyDate").get();//回复时间
                    Long time = Long.valueOf(Pattern.compile("[^0-9]").matcher(timeStr).replaceAll(" ").replaceAll("\\s+", " ").trim().split(" ")[0]);
                    String user1 = new Json(node.get()).jsonPath(".RMemberName").get();//回复用户
                    String userId1 = new Json(node.get()).jsonPath(".RMemberId").get();//回复用户id
                    String commentId1 = new Json(node.get()).jsonPath(".ReplyId").get();//回复评论id
                    String commentId2 = new Json(node.get()).jsonPath(".Quote.ReplyId").get();//原评论id
                    String commentId = commentId1 + commentId2;
                    String content2 = new Json(node.get()).jsonPath(".Quote.RContent").get();//原评论内容
                    String user2 = new Json(node.get()).jsonPath(".Quote.RMemberName").get();//原评论用户
                    String content = "回复：" + content1 + "原评论：" + content2;
                    String user = "回复：" + user1 + "原评论：" + user2;
                    String userId = "回复：" + userId1 + "原评论：" + userId2;
                    String videoUrl = crawlerRecord.tagsCreator().bizTags().getCategoryTag().getKVTagStrVal(AICCommonField.Field_Urls);//文章url
                    String videoId = Pattern.compile("[^0-9]").matcher(videoUrl).replaceAll(" ").replaceAll("\\s+", " ").trim().split(" ")[0];//url的关键字

                    CrawlerData crawlerData = CrawlerData.builder()
                            .data(crawlerRecord, page)
                            .dataId(StringUtils.joinWith("-", crawlerRecord.getDomain(), comment.enumVal(), commentId))
                            .parentId(StringUtils.joinWith("-", crawlerRecord.getDomain(), comment.enumVal(), videoId))
                            .resultLabelTag(CrawlerEnum.CrawlerDataType.valueOf("comment"))
                            .url(videoUrl)
                            .releaseTime(time)
                            .addContentKV(AICCommonField.Field_Content, content)
                            .addContentKV(AICCommonField.Field_Author, user)
                            .addContentKV(AICCommonField.Field_Author_Id, userId)
                            .requestLabelTag(CrawlerEnum.CrawlerRequestType.result)
                            .requestLabelTag(CrawlerEnum.CrawlerRequestType.filter)
                            .flowInPipelineTag("kafka")
                            .build();
                    crawlerData.setFilterPipelineResult(true);
                    crawlerArticleDataList.add(crawlerData);
                } catch (Exception e) {
                    log.error(e.getMessage(), e);
                }
            }
        }
        return crawlerArticleDataList;
    }

    @Override
    public void afterExecute(CrawlerRecordContext context) {
    }

    /*
     * 重新下载
     * */
    private void requestAgainCrawlerRecord(List<CrawlerRequestRecord> crawlerRequestRecords, CrawlerRequestRecord crawlerRecord) {
        CrawlerBusinessTags crawlerBusinessTags = crawlerRecord.tagsCreator().bizTags();
        int count = 0;
        if (crawlerBusinessTags.hasKVTag(REQUEST_AGAIN_TAG)) {
            count = (int) crawlerBusinessTags.getCategoryTag().getKVTag(REQUEST_AGAIN_TAG).getVal();
            if (count >= 20) {
                log.error("pcauto download page the number of retries exceeds the limit" +
                        ",request url {}", crawlerRecord.getHttpRequest().getUrl());
                return;
            }
        }

        String requestUrl = crawlerRecord.getHttpRequest().getUrl();
        CrawlerRequestRecord crawlerRequestRecord = null;
        LabelTag type = crawlerRecord.tagsCreator().requestTags().getCategoryTag().getLabelTags().get("turn_page_item_request");
        if (type == null) {
            crawlerRequestRecord = CrawlerRequestRecord.builder()
                    .turnPageRequest(crawlerRecord)
                    .httpUrl(requestUrl)
                    .recordKey(crawlerRecord.getRecordKey() + count)
                    .releaseTime(crawlerRecord.getReleaseTime())
                    .notFilterRecord()
                    .copyBizTags()
                    .copyResultTags()
                    .build();
        } else {
            crawlerRequestRecord = CrawlerRequestRecord.builder()
                    .itemPageRequest(crawlerRecord)
                    .httpUrl(requestUrl)
                    .recordKey(crawlerRecord.getRecordKey() + count)
                    .releaseTime(crawlerRecord.getReleaseTime())
                    .copyBizTags()
                    .copyResultTags()
                    .build();
        }

        crawlerRequestRecord.setNeedWashPage(crawlerRecord.isNeedWashPage());
        crawlerRequestRecord.setNeedParsedPage(crawlerRecord.isNeedParsedPage());
        crawlerRequestRecord.getHttpRequest().setHeaders(crawlerRecord.getHttpRequest().getHeaders());
        crawlerRequestRecord.getHttpRequest().setExtras(crawlerRecord.getHttpRequest().getExtras());
        crawlerRequestRecord.tagsCreator().bizTags().addCustomKV(REQUEST_AGAIN_TAG, ++count);
        crawlerRequestRecords.add(crawlerRequestRecord);
    }

    private String numberBefore(String number) {
        if (number == null) {
            log.error(DOMAIN + " 传入的粉丝数错误,可能是连接超时原因");
            return "0";
        }
        String substring = number.substring(0, number.length() - 2).trim();
        if (substring.contains("万")) {
            String sub = substring.substring(0, substring.length() - 1).trim();
            String v = (int) Double.parseDouble(sub) * 10000 + "";
            return v;
        } else if (substring.contains("亿")) {
            String sub = substring.substring(0, substring.length() - 1).trim();
            String v = (int) Double.parseDouble(sub) * 100000000 + "";
            return v;
        } else {
            String v = substring;
            return v;
        }
    }
}
